{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "25b3e3e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: openai in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (1.41.1)\n",
      "Requirement already satisfied: pymongo in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (4.8.0)\n",
      "Requirement already satisfied: numpy in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (1.26.4)\n",
      "Requirement already satisfied: pandas in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (2.2.2)\n",
      "Requirement already satisfied: s3fs in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (2024.6.1)\n",
      "Requirement already satisfied: langchain-mongodb in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (0.1.8)\n",
      "Requirement already satisfied: langchain in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (0.2.14)\n",
      "Requirement already satisfied: langchain-community in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (0.2.12)\n",
      "Requirement already satisfied: langchain-openai in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (0.1.22)\n",
      "Requirement already satisfied: python-dotenv in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (1.0.1)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from openai) (4.4.0)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from openai) (1.9.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from openai) (0.27.0)\n",
      "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from openai) (0.5.0)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from openai) (2.8.2)\n",
      "Requirement already satisfied: sniffio in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from openai) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from openai) (4.66.5)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from openai) (4.12.2)\n",
      "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from pymongo) (2.6.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: aiobotocore<3.0.0,>=2.5.4 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from s3fs) (2.13.2)\n",
      "Requirement already satisfied: fsspec==2024.6.1.* in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from s3fs) (2024.6.1)\n",
      "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from s3fs) (3.10.5)\n",
      "Requirement already satisfied: langchain-core<0.3.0,>=0.2.21 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain-mongodb) (0.2.33)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain) (6.0.2)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain) (2.0.32)\n",
      "Requirement already satisfied: langchain-text-splitters<0.3.0,>=0.2.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain) (0.2.2)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain) (0.1.99)\n",
      "Requirement already satisfied: requests<3,>=2 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain) (2.32.3)\n",
      "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.1.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain) (8.5.0)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain-community) (0.6.7)\n",
      "Requirement already satisfied: tiktoken<1,>=0.7 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain-openai) (0.7.0)\n",
      "Collecting botocore<1.34.132,>=1.34.70 (from aiobotocore<3.0.0,>=2.5.4->s3fs)\n",
      "  Obtaining dependency information for botocore<1.34.132,>=1.34.70 from https://files.pythonhosted.org/packages/46/1a/01785fad12a9b1dbeffebd97cd226ea5923114057c64a610dd4eb8a28c7b/botocore-1.34.131-py3-none-any.whl.metadata\n",
      "  Using cached botocore-1.34.131-py3-none-any.whl.metadata (5.7 kB)\n",
      "Requirement already satisfied: wrapt<2.0.0,>=1.10.10 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs) (1.16.0)\n",
      "Requirement already satisfied: aioitertools<1.0.0,>=0.5.1 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs) (0.11.0)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (2.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (24.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs) (1.9.4)\n",
      "Requirement already satisfied: idna>=2.8 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community) (3.21.3)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community) (0.9.0)\n",
      "Requirement already satisfied: certifi in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai) (2024.7.4)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain-core<0.3.0,>=0.2.21->langchain-mongodb) (1.33)\n",
      "Requirement already satisfied: packaging<25,>=23.2 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langchain-core<0.3.0,>=0.2.21->langchain-mongodb) (24.1)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (3.10.7)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.20.1 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai) (2.20.1)\n",
      "Requirement already satisfied: six>=1.5 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from requests<3,>=2->langchain) (3.3.2)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from requests<3,>=2->langchain) (2.2.2)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from tiktoken<1,>=0.7->langchain-openai) (2024.7.24)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from botocore<1.34.132,>=1.34.70->aiobotocore<3.0.0,>=2.5.4->s3fs) (1.0.1)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.3.0,>=0.2.21->langchain-mongodb) (3.0.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/ashwin.gangadhar/mdb-book/venv/lib/python3.11/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain-community) (1.0.0)\n",
      "Using cached botocore-1.34.131-py3-none-any.whl (12.3 MB)\n",
      "Installing collected packages: botocore\n",
      "  Attempting uninstall: botocore\n",
      "    Found existing installation: botocore 1.35.2\n",
      "    Uninstalling botocore-1.35.2:\n",
      "      Successfully uninstalled botocore-1.35.2\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "boto3 1.35.2 requires botocore<1.36.0,>=1.35.2, but you have botocore 1.34.131 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed botocore-1.34.131\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# you can ignore this step if you ran pip install requirements while setting up the python virtual env\n",
    "!pip install openai pymongo numpy pandas s3fs langchain-mongodb langchain langchain-community langchain-openai python-dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0b5e6c2c-0743-4258-baa2-de6165a31540",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import os\n",
    "from getpass import getpass\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "if 'OPENAI_API_KEY' not in os.environ:\n",
    "    os.environ['OPENAI_API_KEY'] = getpass(\"Input Open AI API KEY:\")\n",
    "\n",
    "if 'MONGODB_CONNECTION_STR' not in os.environ:\n",
    "    os.environ['MONGODB_CONNECTION_STRING'] = getpass(\"Input MongoDB Connection String:\")\n",
    "\n",
    "response = openai.embeddings.create(\n",
    "  input=\"Educative answers section is helpful\",\n",
    "  model=\"text-embedding-ada-002\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "cdc70e7a-2e85-498b-aa7b-59e7d65f329d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overview</th>\n",
       "      <th>title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>adult</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Led by Woody, Andy's toys live happily in his ...</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>1995-10-30</td>\n",
       "      <td>7.7</td>\n",
       "      <td>5415</td>\n",
       "      <td>False</td>\n",
       "      <td>1995</td>\n",
       "      <td>10</td>\n",
       "      <td>30</td>\n",
       "      <td>Title: Toy Story  Genres: Animation,Comedy,Fam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>When siblings Judy and Peter discover an encha...</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>1995-12-15</td>\n",
       "      <td>6.9</td>\n",
       "      <td>2413</td>\n",
       "      <td>False</td>\n",
       "      <td>1995</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>Title: Jumanji  Genres: Animation,Comedy,Famil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A family wedding reignites the ancient feud be...</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>1995-12-22</td>\n",
       "      <td>6.5</td>\n",
       "      <td>92</td>\n",
       "      <td>False</td>\n",
       "      <td>1995</td>\n",
       "      <td>12</td>\n",
       "      <td>22</td>\n",
       "      <td>Title: Grumpier Old Men  Genres: Animation,Com...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            overview             title  \\\n",
       "0  Led by Woody, Andy's toys live happily in his ...         Toy Story   \n",
       "1  When siblings Judy and Peter discover an encha...           Jumanji   \n",
       "2  A family wedding reignites the ancient feud be...  Grumpier Old Men   \n",
       "\n",
       "  release_date  vote_average  vote_count  adult  year  month  day  \\\n",
       "0   1995-10-30           7.7        5415  False  1995     10   30   \n",
       "1   1995-12-15           6.9        2413  False  1995     12   15   \n",
       "2   1995-12-22           6.5          92  False  1995     12   22   \n",
       "\n",
       "                                                text  \n",
       "0  Title: Toy Story  Genres: Animation,Comedy,Fam...  \n",
       "1  Title: Jumanji  Genres: Animation,Comedy,Famil...  \n",
       "2  Title: Grumpier Old Men  Genres: Animation,Com...  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import s3fs\n",
    "s3_uri = \"s3://ashwin-partner-bucket/movies_sample_dataset.jsonl\"\n",
    "df = pd.read_json(s3_uri, orient=\"records\", lines=True)\n",
    "df.to_json(\"./movies_sample_dataset.jsonl\", orient=\"records\", lines=True)\n",
    "df[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "9d601c77-f585-4789-b8af-944e057896cb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [07:03<00:00,  4.24s/it]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "df['final'] = df['text'] + \"    Overview: \" + df['overview']\n",
    "df['final'][:5]\n",
    "step =int(np.ceil(df['final'].shape[0]/100))\n",
    "embeddings_t = []\n",
    "lines = []\n",
    "for x,y in list(map(lambda x:(x,x+step), list(range(0,df.shape[0],step)))):\n",
    "    lines += [df.final.values[x:y].tolist()]\n",
    "for i in tqdm(lines):\n",
    "    embeddings_t += openai.embeddings.create(model='text-embedding-ada-002', input=i).data\n",
    "    \n",
    "out = []\n",
    "for ele in embeddings_t:\n",
    "    out += [ele.embedding]\n",
    "df['embedding'] = out\n",
    "df[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "7f8bdcf6-d38c-4034-a104-0cc3caca9172",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overview</th>\n",
       "      <th>title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>adult</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>text</th>\n",
       "      <th>final</th>\n",
       "      <th>embedding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Led by Woody, Andy's toys live happily in his ...</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>1995-10-30</td>\n",
       "      <td>7.7</td>\n",
       "      <td>5415</td>\n",
       "      <td>False</td>\n",
       "      <td>1995</td>\n",
       "      <td>10</td>\n",
       "      <td>30</td>\n",
       "      <td>Title: Toy Story  Genres: Animation,Comedy,Fam...</td>\n",
       "      <td>Title: Toy Story  Genres: Animation,Comedy,Fam...</td>\n",
       "      <td>[-0.012458283454179764, -0.042695507407188416,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>When siblings Judy and Peter discover an encha...</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>1995-12-15</td>\n",
       "      <td>6.9</td>\n",
       "      <td>2413</td>\n",
       "      <td>False</td>\n",
       "      <td>1995</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>Title: Jumanji  Genres: Animation,Comedy,Famil...</td>\n",
       "      <td>Title: Jumanji  Genres: Animation,Comedy,Famil...</td>\n",
       "      <td>[0.015389477834105492, -0.028312528505921364, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A family wedding reignites the ancient feud be...</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>1995-12-22</td>\n",
       "      <td>6.5</td>\n",
       "      <td>92</td>\n",
       "      <td>False</td>\n",
       "      <td>1995</td>\n",
       "      <td>12</td>\n",
       "      <td>22</td>\n",
       "      <td>Title: Grumpier Old Men  Genres: Animation,Com...</td>\n",
       "      <td>Title: Grumpier Old Men  Genres: Animation,Com...</td>\n",
       "      <td>[0.016336416825652122, -0.01960906572639942, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cheated on, mistreated and stepped on, the wom...</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>1995-12-22</td>\n",
       "      <td>6.1</td>\n",
       "      <td>34</td>\n",
       "      <td>False</td>\n",
       "      <td>1995</td>\n",
       "      <td>12</td>\n",
       "      <td>22</td>\n",
       "      <td>Title: Waiting to Exhale  Genres: Animation,Co...</td>\n",
       "      <td>Title: Waiting to Exhale  Genres: Animation,Co...</td>\n",
       "      <td>[-0.018749399110674858, -0.0272560715675354, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Just when George Banks has recovered from his ...</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>1995-02-10</td>\n",
       "      <td>5.7</td>\n",
       "      <td>173</td>\n",
       "      <td>False</td>\n",
       "      <td>1995</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>Title: Father of the Bride Part II  Genres: An...</td>\n",
       "      <td>Title: Father of the Bride Part II  Genres: An...</td>\n",
       "      <td>[0.00420904066413641, -0.020869411528110504, -...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            overview  \\\n",
       "0  Led by Woody, Andy's toys live happily in his ...   \n",
       "1  When siblings Judy and Peter discover an encha...   \n",
       "2  A family wedding reignites the ancient feud be...   \n",
       "3  Cheated on, mistreated and stepped on, the wom...   \n",
       "4  Just when George Banks has recovered from his ...   \n",
       "\n",
       "                         title release_date  vote_average  vote_count  adult  \\\n",
       "0                    Toy Story   1995-10-30           7.7        5415  False   \n",
       "1                      Jumanji   1995-12-15           6.9        2413  False   \n",
       "2             Grumpier Old Men   1995-12-22           6.5          92  False   \n",
       "3            Waiting to Exhale   1995-12-22           6.1          34  False   \n",
       "4  Father of the Bride Part II   1995-02-10           5.7         173  False   \n",
       "\n",
       "   year  month  day                                               text  \\\n",
       "0  1995     10   30  Title: Toy Story  Genres: Animation,Comedy,Fam...   \n",
       "1  1995     12   15  Title: Jumanji  Genres: Animation,Comedy,Famil...   \n",
       "2  1995     12   22  Title: Grumpier Old Men  Genres: Animation,Com...   \n",
       "3  1995     12   22  Title: Waiting to Exhale  Genres: Animation,Co...   \n",
       "4  1995      2   10  Title: Father of the Bride Part II  Genres: An...   \n",
       "\n",
       "                                               final  \\\n",
       "0  Title: Toy Story  Genres: Animation,Comedy,Fam...   \n",
       "1  Title: Jumanji  Genres: Animation,Comedy,Famil...   \n",
       "2  Title: Grumpier Old Men  Genres: Animation,Com...   \n",
       "3  Title: Waiting to Exhale  Genres: Animation,Co...   \n",
       "4  Title: Father of the Bride Part II  Genres: An...   \n",
       "\n",
       "                                           embedding  \n",
       "0  [-0.012458283454179764, -0.042695507407188416,...  \n",
       "1  [0.015389477834105492, -0.028312528505921364, ...  \n",
       "2  [0.016336416825652122, -0.01960906572639942, 0...  \n",
       "3  [-0.018749399110674858, -0.0272560715675354, 0...  \n",
       "4  [0.00420904066413641, -0.020869411528110504, -...  "
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4bff6818-dfb1-4b57-afa9-5f20f08dc5d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "import certifi\n",
    "mongo_client = MongoClient(os.environ[\"MONGODB_CONNECTION_STR\"], tlsCAFile=certifi.where())\n",
    "# Upload documents along with vector embeddings to MongoDB Atlas Collection\n",
    "output_collection = mongo_client[\"sample_movies\"][\"embed_movies\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44a57c5c-db40-4c06-ac13-ee5386e86d59",
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove any pre loaded data\n",
    "if output_collection.count_documents({})>0:\n",
    "    output_collection.delete_many({})\n",
    "e = output_collection.insert_many(df.to_dict(\"records\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "78c5a35c-2e59-4cff-8b70-bc535723c4fc",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': '6627d794b8d9f774f4b2619d',\n",
       "  'name': 'default',\n",
       "  'type': 'vectorSearch',\n",
       "  'status': 'READY',\n",
       "  'queryable': True,\n",
       "  'latestDefinitionVersion': {'version': 0,\n",
       "   'createdAt': datetime.datetime(2024, 4, 23, 15, 45, 24, 356000)},\n",
       "  'latestDefinition': {'fields': [{'type': 'vector',\n",
       "     'numDimensions': 1536,\n",
       "     'path': 'embedding',\n",
       "     'similarity': 'cosine'},\n",
       "    {'type': 'filter', 'path': 'year'}]},\n",
       "  'statusDetail': [{'hostname': 'atlas-uuvcjy-shard-00-01',\n",
       "    'status': 'READY',\n",
       "    'queryable': True,\n",
       "    'mainIndex': {'status': 'READY',\n",
       "     'queryable': True,\n",
       "     'definitionVersion': {'version': 0,\n",
       "      'createdAt': datetime.datetime(2024, 4, 23, 15, 45, 24)},\n",
       "     'definition': {'fields': [{'type': 'vector',\n",
       "        'path': 'embedding',\n",
       "        'numDimensions': 1536,\n",
       "        'similarity': 'cosine'},\n",
       "       {'type': 'filter', 'path': 'year'}]}}},\n",
       "   {'hostname': 'atlas-uuvcjy-shard-00-02',\n",
       "    'status': 'READY',\n",
       "    'queryable': True,\n",
       "    'mainIndex': {'status': 'READY',\n",
       "     'queryable': True,\n",
       "     'definitionVersion': {'version': 0,\n",
       "      'createdAt': datetime.datetime(2024, 4, 23, 15, 45, 24)},\n",
       "     'definition': {'fields': [{'type': 'vector',\n",
       "        'path': 'embedding',\n",
       "        'numDimensions': 1536,\n",
       "        'similarity': 'cosine'},\n",
       "       {'type': 'filter', 'path': 'year'}]}}},\n",
       "   {'hostname': 'atlas-uuvcjy-shard-00-00',\n",
       "    'status': 'READY',\n",
       "    'queryable': True,\n",
       "    'mainIndex': {'status': 'READY',\n",
       "     'queryable': True,\n",
       "     'definitionVersion': {'version': 0,\n",
       "      'createdAt': datetime.datetime(2024, 4, 23, 15, 45, 24)},\n",
       "     'definition': {'fields': [{'type': 'vector',\n",
       "        'path': 'embedding',\n",
       "        'numDimensions': 1536,\n",
       "        'similarity': 'cosine'},\n",
       "       {'type': 'filter', 'path': 'year'}]}}}]}]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(output_collection.list_search_indexes())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b4256a0",
   "metadata": {},
   "source": [
    "# Semantic Similarity Search Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "3172bec5-46e4-4929-88fd-65597d706dfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_vector_search(q, prefilter = {}, postfilter = {},path=\"embedding\",topK=2):\n",
    "    ele = openai.embeddings.create(model='text-embedding-ada-002', input=q).data\n",
    "    query_embedding = ele[0].embedding\n",
    "    vs_query = {\n",
    "                \"index\": \"default\",\n",
    "                \"path\": path,\n",
    "                \"queryVector\": query_embedding,\n",
    "                \"numCandidates\": 10,\n",
    "                \"limit\": topK,\n",
    "            }\n",
    "    if len(prefilter)>0:\n",
    "        vs_query[\"filter\"] = prefilter\n",
    "    new_search_query = {\"$vectorSearch\": vs_query}\n",
    "    project = {\"$project\": {\"score\": {\"$meta\": \"vectorSearchScore\"},\"_id\": 0,\"title\": 1, \"release_date\": 1, \"final\": 1,\"year\": 1}}\n",
    "    if len(postfilter.keys())>0:\n",
    "        postFilter = {\"$match\":postfilter}\n",
    "        res = list(output_collection.aggregate([new_search_query, project, postFilter]))\n",
    "    else:\n",
    "        res = list(output_collection.aggregate([new_search_query, project]))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "id": "8bd18714-6957-455f-8505-c7dbf50b15d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'title': \"Christmas Vacation '91\",\n",
       "  'release_date': '1991-12-20',\n",
       "  'year': 1991,\n",
       "  'final': \"Title: Christmas Vacation '91  Genres: Animation,Comedy,FamilyThis coarse bedroom farce takes place at the St. Moritz ski resort over a Christmas vacation. Among the couples whose lives intersect are a widowed artist honeymooning with his second wife, a gay man traveling with his son and his lover (and hiding each from the other), a snobbish couple from Milan who have been forced to share a suite with a pair of crass Romans, etc.    Overview: This coarse bedroom farce takes place at the St. Moritz ski resort over a Christmas vacation. Among the couples whose lives intersect are a widowed artist honeymooning with his second wife, a gay man traveling with his son and his lover (and hiding each from the other), a snobbish couple from Milan who have been forced to share a suite with a pair of crass Romans, etc.\",\n",
       "  'score': 0.9068390727043152},\n",
       " {'title': 'Happy Christmas',\n",
       "  'release_date': '2014-06-26',\n",
       "  'year': 2014,\n",
       "  'final': 'Title: Happy Christmas  Genres: Animation,Comedy,FamilyAfter a breakup with her boyfriend, a young woman moves in with her older brother, his wife, and their 2-year-old son.    Overview: After a breakup with her boyfriend, a young woman moves in with her older brother, his wife, and their 2-year-old son.',\n",
       "  'score': 0.9064679145812988},\n",
       " {'title': 'Almost Christmas',\n",
       "  'release_date': '2016-11-11',\n",
       "  'year': 2016,\n",
       "  'final': 'Title: Almost Christmas  Genres: Animation,Comedy,FamilyA dysfunctional family gathers together for their first Christmas since their mom died.    Overview: A dysfunctional family gathers together for their first Christmas since their mom died.',\n",
       "  'score': 0.9042526483535767},\n",
       " {'title': 'Finding Christmas',\n",
       "  'release_date': '2013-12-15',\n",
       "  'year': 2013,\n",
       "  'final': \"Title: Finding Christmas  Genres: Animation,Comedy,FamilySingle mother Ryan has just about given up on dating after her divorce, happily accepting her young son as the most important man in her life. That all changes when Ryan's brother Owen, also feeling unlucky in love after a bad breakup, swaps his home in their small North Carolina town with New York City adman Sean.    Overview: Single mother Ryan has just about given up on dating after her divorce, happily accepting her young son as the most important man in her life. That all changes when Ryan's brother Owen, also feeling unlucky in love after a bad breakup, swaps his home in their small North Carolina town with New York City adman Sean.\",\n",
       "  'score': 0.9033454656600952},\n",
       " {'title': 'A Christmas Story 2',\n",
       "  'release_date': '2012-09-06',\n",
       "  'year': 2012,\n",
       "  'final': 'Title: A Christmas Story 2  Genres: Animation,Comedy,FamilyThe original traditional one-hundred-percent red-blooded two-fisted all-american christmas contiunues five years later with Ralphie, Randy mom and the old man. This time Ralphie has his eyes fixed on a car. But trouble is sure to follow.    Overview: The original traditional one-hundred-percent red-blooded two-fisted all-american christmas contiunues five years later with Ralphie, Randy mom and the old man. This time Ralphie has his eyes fixed on a car. But trouble is sure to follow.',\n",
       "  'score': 0.9022564888000488}]"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_vector_search(\"I like Christmas movies, any recommendations for movies release after 1990?\", prefilter={\"year\": {\"$gt\": 1990}}, topK=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "id": "7ece2a35-bf29-4b9c-bfe1-10a7d68e6cee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'title': \"Christmas Vacation '91\",\n",
       "  'release_date': '1991-12-20',\n",
       "  'year': 1991,\n",
       "  'final': \"Title: Christmas Vacation '91  Genres: Animation,Comedy,FamilyThis coarse bedroom farce takes place at the St. Moritz ski resort over a Christmas vacation. Among the couples whose lives intersect are a widowed artist honeymooning with his second wife, a gay man traveling with his son and his lover (and hiding each from the other), a snobbish couple from Milan who have been forced to share a suite with a pair of crass Romans, etc.    Overview: This coarse bedroom farce takes place at the St. Moritz ski resort over a Christmas vacation. Among the couples whose lives intersect are a widowed artist honeymooning with his second wife, a gay man traveling with his son and his lover (and hiding each from the other), a snobbish couple from Milan who have been forced to share a suite with a pair of crass Romans, etc.\",\n",
       "  'score': 0.9068360328674316},\n",
       " {'title': 'Happy Christmas',\n",
       "  'release_date': '2014-06-26',\n",
       "  'year': 2014,\n",
       "  'final': 'Title: Happy Christmas  Genres: Animation,Comedy,FamilyAfter a breakup with her boyfriend, a young woman moves in with her older brother, his wife, and their 2-year-old son.    Overview: After a breakup with her boyfriend, a young woman moves in with her older brother, his wife, and their 2-year-old son.',\n",
       "  'score': 0.9064450263977051}]"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_vector_search(\"I like Christmas movies, any recommendations for movies release after 1990?\", prefilter={\"year\":{\"$gt\": 1990}}, postfilter={\"score\": {\"$gt\":0.905}},topK=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39ca043e",
   "metadata": {},
   "source": [
    "# Simple/Naive RAG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "deefb17f-1553-42d5-a359-0560f4291829",
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "client = OpenAI()\n",
    "def invoke_llm(prompt, model_name='gpt-3.5-turbo-0125'):\n",
    "    \"\"\"\n",
    "    Queries with input prompt to OpenAI API using the chat completion api gets the model's response.\n",
    "    \"\"\"\n",
    "    \n",
    "    response = client.chat.completions.create(\n",
    "      model=model_name,\n",
    "      messages=[\n",
    "        {\n",
    "          \"role\": \"user\",\n",
    "          \"content\": prompt\n",
    "        }\n",
    "      ],\n",
    "      temperature=0.2,\n",
    "      max_tokens=256,\n",
    "      top_p=1,\n",
    "      frequency_penalty=0,\n",
    "      presence_penalty=0\n",
    "    )\n",
    "    \n",
    "    chatbot_response = response.choices[0].message.content.strip()\n",
    "    return chatbot_response\n",
    "invoke_llm(\"This is a test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "63513651-8662-4ad8-8981-3c3136a5f70e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prompt(question, context):\n",
    "    prompt = f\"\"\"Question: {question}\n",
    "            System: Let's think step by step.\n",
    "            Context: {context} \n",
    "            \"\"\"\n",
    "    return prompt\n",
    "def get_recommendation_prompt(query, context):\n",
    "    prompt = f\"\"\"\n",
    "        From the given movie listing data, choose a few great movie recommendations.\n",
    "        User query: {query}\n",
    "        Context: {context} \n",
    "        \n",
    "        Movie Recommendations:\n",
    "        1. Movie_name: Movie_overview\n",
    "        \"\"\"\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "id": "f4258ae2-3c85-4280-a95e-a40f9e8a6951",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The movie in which the footballer goes completely blind is \"23 Blast.\"'"
      ]
     },
     "execution_count": 297,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idea = \"In which movie does the footballer goes completly blind?\"\n",
    "search_response = query_vector_search(idea, prefilter={\"year\":{\"$gt\": 1990}}, postfilter={\"score\": {\"$gt\":0.8}},topK=10)\n",
    "premise = \"\\n\".join(list(map(lambda x:x['final'], search_response)))\n",
    "prompt = get_prompt(idea, premise)\n",
    "invoke_llm(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "id": "85230c66-a8c0-427a-87ec-f81aa85d79a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. Happy Christmas: After a breakup with her boyfriend, a young woman moves in with her older brother, his wife, and their 2-year-old son.\n",
      "2. Almost Christmas: A dysfunctional family gathers together for their first Christmas since their mom died.\n",
      "3. Finding Christmas: Single mother Ryan's dating life changes when her brother swaps homes with a New York City adman.\n",
      "4. Christmas Eve: Hilarity, romance, and transcendence prevail after a power outage traps six different groups of New Yorkers inside elevators on Christmas Eve.\n",
      "5. National Lampoon's Christmas Vacation: The Griswolds prepare for a family seasonal celebration, but things never run smoothly for Clark, his wife Ellen, and their two kids.\n"
     ]
    }
   ],
   "source": [
    "question = \"I like Christmas movies, any recommendations for movies release after 1990?\"\n",
    "search_response = query_vector_search(question,topK=10)\n",
    "context = \"\\n\".join(list(map(lambda x:x['final'], search_response)))\n",
    "print(invoke_llm(get_recommendation_prompt(\"I like Christmas movies, any recommendations for movies release after 1990?\", context)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "id": "dfa42e46-e514-48a2-bc6f-2a5191a8789f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The movie you are referring to is \"The Damned United\" (2009), where the character Brian Clough, a former footballer and manager, goes temporarily blind due to stress and anxiety.\n"
     ]
    }
   ],
   "source": [
    "print(invoke_llm(\"In which movie does a footballer go completely blind?\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "id": "9bd3edd8-9690-4cc0-9693-5dbd79ca809c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The movie in which a footballer goes completely blind is \"23 Blast.\"\n"
     ]
    }
   ],
   "source": [
    "idea = \"In which movie does a footballer go completely blind?\"\n",
    "search_response = query_vector_search(idea, prefilter={\"year\":{\"$gt\": 1990}}, postfilter={\"score\": {\"$gt\":0.8}},topK=10)\n",
    "premise = \"\\n\".join(list(map(lambda x:x['final'], search_response)))\n",
    "print(invoke_llm(get_prompt(idea, premise)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d73a1a2-b4e5-49a2-96ef-2ed29687d304",
   "metadata": {},
   "source": [
    "# Semantic Similarity with Langchain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "id": "6d6d7c2b-927a-4fff-a02b-9f613578e118",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      " \"title\": \"Christmas Vacation '91\",\n",
      " \"year\": 1991,\n",
      " \"final\": \"Title: Christmas Vacation '91  Genres: Animation,Comedy,FamilyThis coarse bedroom farce takes place at the St. Moritz ski resort over a Christmas vacation. Among the couples whose lives intersect are a widowed artist honeymooning with his second wife, a gay man traveling with his son and his lover (and hiding each from the other), a snobbish couple from Milan who have been forced to share a suite with a pair of crass Romans, etc.\"\n",
      "}\n",
      "{\n",
      " \"title\": \"Happy Christmas\",\n",
      " \"year\": 2014,\n",
      " \"final\": \"Title: Happy Christmas  Genres: Animation,Comedy,FamilyAfter a breakup with her boyfriend, a young woman moves in with her older brother, his wife, and their 2-year-old son.\"\n",
      "}\n",
      "{\n",
      " \"title\": \"Almost Christmas\",\n",
      " \"year\": 2016,\n",
      " \"final\": \"Title: Almost Christmas  Genres: Animation,Comedy,FamilyA dysfunctional family gathers together for their first Christmas since their mom died.\"\n",
      "}\n",
      "{\n",
      " \"title\": \"Finding Christmas\",\n",
      " \"year\": 2013,\n",
      " \"final\": \"Title: Finding Christmas  Genres: Animation,Comedy,FamilySingle mother Ryan has just about given up on dating after her divorce, happily accepting her young son as the most important man in her life. That all changes when Ryan's brother Owen, also feeling unlucky in love after a bad breakup, swaps his home in their small North Carolina town with New York City adman Sean.\"\n",
      "}\n",
      "{\n",
      " \"title\": \"A Christmas Story 2\",\n",
      " \"year\": 2012,\n",
      " \"final\": \"Title: A Christmas Story 2  Genres: Animation,Comedy,FamilyThe original traditional one-hundred-percent red-blooded two-fisted all-american christmas contiunues five years later with Ralphie, Randy mom and the old man. This time Ralphie has his eyes fixed on a car. But trouble is sure to follow.\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "from langchain.vectorstores import MongoDBAtlasVectorSearch\n",
    "from langchain.embeddings import OpenAIEmbeddings\n",
    "import json\n",
    "\n",
    "embedding_model = OpenAIEmbeddings()\n",
    "vector_search = MongoDBAtlasVectorSearch(output_collection, embedding_model, text_key='final')\n",
    "fquery = {\"year\": {\"$gt\": 1990}}\n",
    "search_kwargs = {\n",
    "    \"k\": 5,\n",
    "    'filter': fquery,\n",
    "}\n",
    "retriever = vector_search.as_retriever(search_kwargs=search_kwargs)\n",
    "docs = retriever.get_relevant_documents(\"I like Christmas movies, any recommendations for movies release after 1990?\")\n",
    "for doc in docs:\n",
    "    foo = {}\n",
    "    foo['title'] = doc.metadata['title']\n",
    "    foo['year'] = doc.metadata['year']\n",
    "    foo['final'] = doc.metadata['text']\n",
    "    print(json.dumps(foo,indent=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44f99fc1",
   "metadata": {},
   "source": [
    "# Chunking Strategies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "id": "6336efbc-7fbe-49eb-8804-f2907b7aa983",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -qU langchain-text-splitters pypdf2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "id": "a38d7778-135f-40e7-9b78-af3e7fb31980",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Uber \\n2021 \\nAnnual ReportUber’s Mission\\nWe reimagine the way the world moves for the better\\nWe are Uber. The go-getters. The kind of people who are relentless about our mission to help people go anywhere and get anything  and earn their way . \\nMovement is what we  power . It’s our lifeblood. It runs through our veins. It’s what gets us out of bed each morning. It pushes us to constantly reimagine \\nhow we can move better. For you. For all the places you want to go. For all the things you want to get. For all the ways you want to earn. Across the entire \\nworld. In real time . At the incredible speed of now.\\nOur Values\\nOur values reflect who we are and where we’re going. They guide our decision-making, unite and define our culture, and tell a story to the world about \\nUber’s corporate purpose.\\nDo the right thingPeriod.\\nGo get itBring the mindset of a champion.\\nOur ambition is what drives us to achieve our mission. How we define a champion mindset \\nisn't based on how we perform on our best\""
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from PyPDF2 import PdfReader\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "def get_pdf_data(pdf_docs):\n",
    "    for pdf in pdf_docs:\n",
    "        text = \"\"\n",
    "        pdf_reader = PdfReader(pdf)\n",
    "        for page in pdf_reader.pages:\n",
    "            text += page.extract_text()\n",
    "        return text\n",
    "text = get_pdf_data([\"./Uber-Annual-Report.pdf\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "id": "ed35d7e3-52f3-4111-b84f-c742760ed655",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "797"
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    # Set a really small chunk size, just to show.\n",
    "    chunk_size=1000,\n",
    "    chunk_overlap=100,\n",
    "    length_function=len,\n",
    "    is_separator_regex=False,\n",
    ")\n",
    "\n",
    "chunks = text_splitter.split_text(text)\n",
    "len(chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "id": "7c828cf2-2bcb-4af0-9a63-ba6d1361be70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------\n",
      "First Chunk:\n",
      "Uber \n",
      "2021 \n",
      "Annual ReportUber’s Mission\n",
      "We reimagine the way the world moves for the better\n",
      "We are Uber. The go-getters. The kind of people who are relentless about our mission to help people go anywhere and get anything  and earn their way . \n",
      "Movement is what we  power . It’s our lifeblood. It runs through our veins. It’s what gets us out of bed each morning. It pushes us to constantly reimagine \n",
      "how we can move better. For you. For all the places you want to go. For all the things you want to get. For all the ways you want to earn. Across the entire \n",
      "world. In real time . At the incredible speed of now.\n",
      "Our Values\n",
      "Our values reflect who we are and where we’re going. They guide our decision-making, unite and define our culture, and tell a story to the world about \n",
      "Uber’s corporate purpose.\n",
      "Do the right thingPeriod.\n",
      "Go get itBring the mindset of a champion.\n",
      "Our ambition is what drives us to achieve our mission. How we define a champion mindset\n",
      "--------------\n",
      "Second Chunk:\n",
      "Our ambition is what drives us to achieve our mission. How we define a champion mindset \n",
      "isn't based on how we perform on our best days, it's how we respond on our worst days. \n",
      "We hustle, embrace the grind, overcome adversity, and play to win for the people we serve. \n",
      "Because it matters.\n",
      "Trip  \n",
      "obsessedMake magic in the marketplace.\n",
      "The trip is where the marketplace comes to life. The earner, rider, eater, carrier , and \n",
      "merchant are the people who connect in our marketplace - and we see every side. This \n",
      "requires judgment to make difficult trade-offs, blending algorithms with human ingenuity, \n",
      "and the ability to create simplicity from complexity. When we get the balance right for \n",
      "everyone, Uber magic happens. \n",
      "Build with  \n",
      "heartWe care.\n",
      "We work at Uber because our products profoundly affect lives and we care deeply about \n",
      "our impact. Putting ourselves in the shoes of people who connect in our marketplace\n",
      "--------------\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n--------------\")\n",
    "print(\"First Chunk:\\n\"+chunks[0]+ \"\\n--------------\")\n",
    "print(\"Second Chunk:\\n\"+chunks[1]+ \"\\n--------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "id": "ab869373-42fd-45ba-8438-303ae1b0b3cf",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;3m⚠ As of spaCy v3.0, shortcuts like 'en' are deprecated. Please use the\n",
      "full pipeline package name 'en_core_web_sm' instead.\u001b[0m\n",
      "Collecting en-core-web-sm==3.7.1\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl (12.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m27.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: spacy<3.8.0,>=3.7.2 in ./anaconda3/lib/python3.9/site-packages (from en-core-web-sm==3.7.1) (3.7.4)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.0.12)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.0.5)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.0.10)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.0.8)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.0.9)\n",
      "Requirement already satisfied: thinc<8.3.0,>=8.2.2 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (8.2.3)\n",
      "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.1.2)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.4.8)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.0.10)\n",
      "Requirement already satisfied: weasel<0.4.0,>=0.1.0 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.3.4)\n",
      "Requirement already satisfied: typer<0.10.0,>=0.3.0 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.9.0)\n",
      "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (5.2.1)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (4.65.0)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.31.0)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.6.3)\n",
      "Requirement already satisfied: jinja2 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.1.3)\n",
      "Requirement already satisfied: setuptools in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (68.2.2)\n",
      "Requirement already satisfied: packaging>=20.0 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (23.2)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.3.0)\n",
      "Requirement already satisfied: numpy>=1.19.0 in ./anaconda3/lib/python3.9/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.26.3)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in ./anaconda3/lib/python3.9/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.16.3 in ./anaconda3/lib/python3.9/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.16.3)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in ./anaconda3/lib/python3.9/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (4.10.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in ./anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in ./anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in ./anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.26.18)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2024.2.2)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.7.8 in ./anaconda3/lib/python3.9/site-packages (from thinc<8.3.0,>=8.2.2->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.7.11)\n",
      "Requirement already satisfied: confection<1.0.0,>=0.0.1 in ./anaconda3/lib/python3.9/site-packages (from thinc<8.3.0,>=8.2.2->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.1.4)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in ./anaconda3/lib/python3.9/site-packages (from typer<0.10.0,>=0.3.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (8.1.7)\n",
      "Requirement already satisfied: cloudpathlib<0.17.0,>=0.7.0 in ./anaconda3/lib/python3.9/site-packages (from weasel<0.4.0,>=0.1.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in ./anaconda3/lib/python3.9/site-packages (from jinja2->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.1.5)\n",
      "Installing collected packages: en-core-web-sm\n",
      "Successfully installed en-core-web-sm-3.7.1\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('en_core_web_sm')\n"
     ]
    }
   ],
   "source": [
    "!pip install -qU  spacy\n",
    "!python -m spacy download en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "id": "ccf7e3b2-9cba-4da2-be9f-380fec03f1c4",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ashwin.gangadhar/anaconda3/lib/python3.9/site-packages/spacy/pipeline/lemmatizer.py:211: UserWarning: [W108] The rule-based lemmatizer did not find POS annotation for one or more tokens. Check that your pipeline includes components that assign token.pos, typically 'tagger'+'attribute_ruler' or 'morphologizer'.\n",
      "  warnings.warn(Warnings.W108)\n",
      "Created a chunk of size 952, which is longer than the specified 500\n",
      "Created a chunk of size 509, which is longer than the specified 500\n",
      "Created a chunk of size 792, which is longer than the specified 500\n",
      "Created a chunk of size 659, which is longer than the specified 500\n",
      "Created a chunk of size 626, which is longer than the specified 500\n",
      "Created a chunk of size 896, which is longer than the specified 500\n",
      "Created a chunk of size 1024, which is longer than the specified 500\n",
      "Created a chunk of size 1288, which is longer than the specified 500\n",
      "Created a chunk of size 1186, which is longer than the specified 500\n",
      "Created a chunk of size 749, which is longer than the specified 500\n",
      "Created a chunk of size 945, which is longer than the specified 500\n",
      "Created a chunk of size 621, which is longer than the specified 500\n",
      "Created a chunk of size 554, which is longer than the specified 500\n",
      "Created a chunk of size 706, which is longer than the specified 500\n",
      "Created a chunk of size 519, which is longer than the specified 500\n",
      "Created a chunk of size 616, which is longer than the specified 500\n",
      "Created a chunk of size 837, which is longer than the specified 500\n",
      "Created a chunk of size 878, which is longer than the specified 500\n",
      "Created a chunk of size 634, which is longer than the specified 500\n",
      "Created a chunk of size 1804, which is longer than the specified 500\n",
      "Created a chunk of size 880, which is longer than the specified 500\n",
      "Created a chunk of size 660, which is longer than the specified 500\n",
      "Created a chunk of size 518, which is longer than the specified 500\n",
      "Created a chunk of size 514, which is longer than the specified 500\n",
      "Created a chunk of size 776, which is longer than the specified 500\n",
      "Created a chunk of size 817, which is longer than the specified 500\n",
      "Created a chunk of size 1645, which is longer than the specified 500\n",
      "Created a chunk of size 1068, which is longer than the specified 500\n",
      "Created a chunk of size 977, which is longer than the specified 500\n",
      "Created a chunk of size 726, which is longer than the specified 500\n",
      "Created a chunk of size 765, which is longer than the specified 500\n",
      "Created a chunk of size 761, which is longer than the specified 500\n",
      "Created a chunk of size 520, which is longer than the specified 500\n",
      "Created a chunk of size 1022, which is longer than the specified 500\n",
      "Created a chunk of size 528, which is longer than the specified 500\n",
      "Created a chunk of size 1640, which is longer than the specified 500\n",
      "Created a chunk of size 645, which is longer than the specified 500\n",
      "Created a chunk of size 616, which is longer than the specified 500\n",
      "Created a chunk of size 738, which is longer than the specified 500\n",
      "Created a chunk of size 1276, which is longer than the specified 500\n",
      "Created a chunk of size 598, which is longer than the specified 500\n",
      "Created a chunk of size 952, which is longer than the specified 500\n",
      "Created a chunk of size 1495, which is longer than the specified 500\n",
      "Created a chunk of size 884, which is longer than the specified 500\n",
      "Created a chunk of size 547, which is longer than the specified 500\n",
      "Created a chunk of size 740, which is longer than the specified 500\n",
      "Created a chunk of size 523, which is longer than the specified 500\n",
      "Created a chunk of size 1505, which is longer than the specified 500\n",
      "Created a chunk of size 1017, which is longer than the specified 500\n",
      "Created a chunk of size 1791, which is longer than the specified 500\n",
      "Created a chunk of size 1020, which is longer than the specified 500\n",
      "Created a chunk of size 1234, which is longer than the specified 500\n",
      "Created a chunk of size 624, which is longer than the specified 500\n",
      "Created a chunk of size 608, which is longer than the specified 500\n",
      "Created a chunk of size 623, which is longer than the specified 500\n",
      "Created a chunk of size 682, which is longer than the specified 500\n",
      "Created a chunk of size 1492, which is longer than the specified 500\n",
      "Created a chunk of size 668, which is longer than the specified 500\n",
      "Created a chunk of size 538, which is longer than the specified 500\n",
      "Created a chunk of size 847, which is longer than the specified 500\n",
      "Created a chunk of size 1590, which is longer than the specified 500\n",
      "Created a chunk of size 713, which is longer than the specified 500\n",
      "Created a chunk of size 1097, which is longer than the specified 500\n",
      "Created a chunk of size 2684, which is longer than the specified 500\n",
      "Created a chunk of size 578, which is longer than the specified 500\n",
      "Created a chunk of size 678, which is longer than the specified 500\n",
      "Created a chunk of size 773, which is longer than the specified 500\n",
      "Created a chunk of size 1357, which is longer than the specified 500\n",
      "Created a chunk of size 1524, which is longer than the specified 500\n",
      "Created a chunk of size 735, which is longer than the specified 500\n",
      "Created a chunk of size 663, which is longer than the specified 500\n",
      "Created a chunk of size 945, which is longer than the specified 500\n",
      "Created a chunk of size 830, which is longer than the specified 500\n",
      "Created a chunk of size 726, which is longer than the specified 500\n",
      "Created a chunk of size 669, which is longer than the specified 500\n",
      "Created a chunk of size 1134, which is longer than the specified 500\n",
      "Created a chunk of size 587, which is longer than the specified 500\n",
      "Created a chunk of size 1100, which is longer than the specified 500\n",
      "Created a chunk of size 719, which is longer than the specified 500\n",
      "Created a chunk of size 853, which is longer than the specified 500\n",
      "Created a chunk of size 1179, which is longer than the specified 500\n",
      "Created a chunk of size 832, which is longer than the specified 500\n",
      "Created a chunk of size 675, which is longer than the specified 500\n",
      "Created a chunk of size 976, which is longer than the specified 500\n",
      "Created a chunk of size 1210, which is longer than the specified 500\n",
      "Created a chunk of size 976, which is longer than the specified 500\n",
      "Created a chunk of size 924, which is longer than the specified 500\n",
      "Created a chunk of size 676, which is longer than the specified 500\n",
      "Created a chunk of size 1833, which is longer than the specified 500\n",
      "Created a chunk of size 767, which is longer than the specified 500\n",
      "Created a chunk of size 767, which is longer than the specified 500\n",
      "Created a chunk of size 615, which is longer than the specified 500\n",
      "Created a chunk of size 559, which is longer than the specified 500\n",
      "Created a chunk of size 733, which is longer than the specified 500\n",
      "Created a chunk of size 1275, which is longer than the specified 500\n",
      "Created a chunk of size 630, which is longer than the specified 500\n",
      "Created a chunk of size 564, which is longer than the specified 500\n",
      "Created a chunk of size 753, which is longer than the specified 500\n",
      "Created a chunk of size 525, which is longer than the specified 500\n",
      "Created a chunk of size 1377, which is longer than the specified 500\n",
      "Created a chunk of size 1089, which is longer than the specified 500\n",
      "Created a chunk of size 670, which is longer than the specified 500\n",
      "Created a chunk of size 682, which is longer than the specified 500\n",
      "Created a chunk of size 600, which is longer than the specified 500\n",
      "Created a chunk of size 536, which is longer than the specified 500\n",
      "Created a chunk of size 731, which is longer than the specified 500\n",
      "Created a chunk of size 927, which is longer than the specified 500\n",
      "Created a chunk of size 891, which is longer than the specified 500\n",
      "Created a chunk of size 787, which is longer than the specified 500\n",
      "Created a chunk of size 924, which is longer than the specified 500\n",
      "Created a chunk of size 780, which is longer than the specified 500\n",
      "Created a chunk of size 1441, which is longer than the specified 500\n",
      "Created a chunk of size 640, which is longer than the specified 500\n",
      "Created a chunk of size 530, which is longer than the specified 500\n",
      "Created a chunk of size 1242, which is longer than the specified 500\n",
      "Created a chunk of size 564, which is longer than the specified 500\n",
      "Created a chunk of size 995, which is longer than the specified 500\n",
      "Created a chunk of size 545, which is longer than the specified 500\n",
      "Created a chunk of size 794, which is longer than the specified 500\n",
      "Created a chunk of size 793, which is longer than the specified 500\n",
      "Created a chunk of size 947, which is longer than the specified 500\n",
      "Created a chunk of size 543, which is longer than the specified 500\n",
      "Created a chunk of size 2826, which is longer than the specified 500\n",
      "Created a chunk of size 534, which is longer than the specified 500\n",
      "Created a chunk of size 503, which is longer than the specified 500\n",
      "Created a chunk of size 957, which is longer than the specified 500\n",
      "Created a chunk of size 786, which is longer than the specified 500\n",
      "Created a chunk of size 1184, which is longer than the specified 500\n",
      "Created a chunk of size 665, which is longer than the specified 500\n",
      "Created a chunk of size 678, which is longer than the specified 500\n",
      "Created a chunk of size 1070, which is longer than the specified 500\n",
      "Created a chunk of size 521, which is longer than the specified 500\n",
      "Created a chunk of size 663, which is longer than the specified 500\n",
      "Created a chunk of size 522, which is longer than the specified 500\n",
      "Created a chunk of size 612, which is longer than the specified 500\n",
      "Created a chunk of size 922, which is longer than the specified 500\n",
      "Created a chunk of size 2380, which is longer than the specified 500\n",
      "Created a chunk of size 548, which is longer than the specified 500\n",
      "Created a chunk of size 797, which is longer than the specified 500\n",
      "Created a chunk of size 755, which is longer than the specified 500\n",
      "Created a chunk of size 828, which is longer than the specified 500\n",
      "Created a chunk of size 1525, which is longer than the specified 500\n",
      "Created a chunk of size 2597, which is longer than the specified 500\n",
      "Created a chunk of size 1680, which is longer than the specified 500\n",
      "Created a chunk of size 1014, which is longer than the specified 500\n",
      "Created a chunk of size 803, which is longer than the specified 500\n",
      "Created a chunk of size 887, which is longer than the specified 500\n",
      "Created a chunk of size 556, which is longer than the specified 500\n",
      "Created a chunk of size 1235, which is longer than the specified 500\n",
      "Created a chunk of size 749, which is longer than the specified 500\n",
      "Created a chunk of size 609, which is longer than the specified 500\n",
      "Created a chunk of size 754, which is longer than the specified 500\n",
      "Created a chunk of size 855, which is longer than the specified 500\n",
      "Created a chunk of size 557, which is longer than the specified 500\n",
      "Created a chunk of size 1427, which is longer than the specified 500\n",
      "Created a chunk of size 1032, which is longer than the specified 500\n",
      "Created a chunk of size 620, which is longer than the specified 500\n",
      "Created a chunk of size 953, which is longer than the specified 500\n",
      "Created a chunk of size 553, which is longer than the specified 500\n",
      "Created a chunk of size 544, which is longer than the specified 500\n",
      "Created a chunk of size 821, which is longer than the specified 500\n",
      "Created a chunk of size 689, which is longer than the specified 500\n",
      "Created a chunk of size 893, which is longer than the specified 500\n",
      "Created a chunk of size 675, which is longer than the specified 500\n",
      "Created a chunk of size 617, which is longer than the specified 500\n",
      "Created a chunk of size 2045, which is longer than the specified 500\n",
      "Created a chunk of size 1315, which is longer than the specified 500\n",
      "Created a chunk of size 714, which is longer than the specified 500\n",
      "Created a chunk of size 657, which is longer than the specified 500\n",
      "Created a chunk of size 758, which is longer than the specified 500\n",
      "Created a chunk of size 869, which is longer than the specified 500\n",
      "Created a chunk of size 907, which is longer than the specified 500\n",
      "Created a chunk of size 689, which is longer than the specified 500\n",
      "Created a chunk of size 721, which is longer than the specified 500\n",
      "Created a chunk of size 580, which is longer than the specified 500\n",
      "Created a chunk of size 1690, which is longer than the specified 500\n",
      "Created a chunk of size 1115, which is longer than the specified 500\n",
      "Created a chunk of size 749, which is longer than the specified 500\n",
      "Created a chunk of size 592, which is longer than the specified 500\n",
      "Created a chunk of size 611, which is longer than the specified 500\n",
      "Created a chunk of size 530, which is longer than the specified 500\n",
      "Created a chunk of size 1308, which is longer than the specified 500\n",
      "Created a chunk of size 790, which is longer than the specified 500\n",
      "Created a chunk of size 737, which is longer than the specified 500\n",
      "Created a chunk of size 1644, which is longer than the specified 500\n",
      "Created a chunk of size 836, which is longer than the specified 500\n",
      "Created a chunk of size 1038, which is longer than the specified 500\n",
      "Created a chunk of size 695, which is longer than the specified 500\n",
      "Created a chunk of size 670, which is longer than the specified 500\n",
      "Created a chunk of size 1651, which is longer than the specified 500\n",
      "Created a chunk of size 2174, which is longer than the specified 500\n",
      "Created a chunk of size 528, which is longer than the specified 500\n",
      "Created a chunk of size 1110, which is longer than the specified 500\n",
      "Created a chunk of size 586, which is longer than the specified 500\n",
      "Created a chunk of size 1329, which is longer than the specified 500\n",
      "Created a chunk of size 580, which is longer than the specified 500\n",
      "Created a chunk of size 795, which is longer than the specified 500\n",
      "Created a chunk of size 643, which is longer than the specified 500\n",
      "Created a chunk of size 938, which is longer than the specified 500\n",
      "Created a chunk of size 1129, which is longer than the specified 500\n",
      "Created a chunk of size 1112, which is longer than the specified 500\n",
      "Created a chunk of size 634, which is longer than the specified 500\n",
      "Created a chunk of size 638, which is longer than the specified 500\n",
      "Created a chunk of size 1064, which is longer than the specified 500\n",
      "Created a chunk of size 524, which is longer than the specified 500\n",
      "Created a chunk of size 796, which is longer than the specified 500\n",
      "Created a chunk of size 815, which is longer than the specified 500\n",
      "Created a chunk of size 1055, which is longer than the specified 500\n",
      "Created a chunk of size 713, which is longer than the specified 500\n",
      "Created a chunk of size 922, which is longer than the specified 500\n",
      "Created a chunk of size 799, which is longer than the specified 500\n",
      "Created a chunk of size 576, which is longer than the specified 500\n",
      "Created a chunk of size 757, which is longer than the specified 500\n",
      "Created a chunk of size 2029, which is longer than the specified 500\n",
      "Created a chunk of size 1182, which is longer than the specified 500\n",
      "Created a chunk of size 636, which is longer than the specified 500\n",
      "Created a chunk of size 940, which is longer than the specified 500\n",
      "Created a chunk of size 2026, which is longer than the specified 500\n",
      "Created a chunk of size 1508, which is longer than the specified 500\n",
      "Created a chunk of size 1328, which is longer than the specified 500\n",
      "Created a chunk of size 706, which is longer than the specified 500\n",
      "Created a chunk of size 684, which is longer than the specified 500\n",
      "Created a chunk of size 717, which is longer than the specified 500\n",
      "Created a chunk of size 537, which is longer than the specified 500\n",
      "Created a chunk of size 1080, which is longer than the specified 500\n",
      "Created a chunk of size 585, which is longer than the specified 500\n",
      "Created a chunk of size 926, which is longer than the specified 500\n",
      "Created a chunk of size 565, which is longer than the specified 500\n",
      "Created a chunk of size 573, which is longer than the specified 500\n",
      "Created a chunk of size 770, which is longer than the specified 500\n",
      "Created a chunk of size 604, which is longer than the specified 500\n",
      "Created a chunk of size 1286, which is longer than the specified 500\n",
      "Created a chunk of size 655, which is longer than the specified 500\n",
      "Created a chunk of size 903, which is longer than the specified 500\n",
      "Created a chunk of size 748, which is longer than the specified 500\n",
      "Created a chunk of size 725, which is longer than the specified 500\n",
      "Created a chunk of size 685, which is longer than the specified 500\n",
      "Created a chunk of size 713, which is longer than the specified 500\n",
      "Created a chunk of size 538, which is longer than the specified 500\n",
      "Created a chunk of size 821, which is longer than the specified 500\n",
      "Created a chunk of size 1176, which is longer than the specified 500\n",
      "Created a chunk of size 530, which is longer than the specified 500\n",
      "Created a chunk of size 509, which is longer than the specified 500\n",
      "Created a chunk of size 1021, which is longer than the specified 500\n",
      "Created a chunk of size 764, which is longer than the specified 500\n",
      "Created a chunk of size 1362, which is longer than the specified 500\n",
      "Created a chunk of size 578, which is longer than the specified 500\n",
      "Created a chunk of size 642, which is longer than the specified 500\n",
      "Created a chunk of size 961, which is longer than the specified 500\n",
      "Created a chunk of size 1231, which is longer than the specified 500\n",
      "Created a chunk of size 530, which is longer than the specified 500\n",
      "Created a chunk of size 702, which is longer than the specified 500\n",
      "Created a chunk of size 620, which is longer than the specified 500\n",
      "Created a chunk of size 894, which is longer than the specified 500\n",
      "Created a chunk of size 606, which is longer than the specified 500\n",
      "Created a chunk of size 955, which is longer than the specified 500\n",
      "Created a chunk of size 543, which is longer than the specified 500\n",
      "Created a chunk of size 754, which is longer than the specified 500\n",
      "Created a chunk of size 828, which is longer than the specified 500\n",
      "Created a chunk of size 1453, which is longer than the specified 500\n",
      "Created a chunk of size 571, which is longer than the specified 500\n",
      "Created a chunk of size 625, which is longer than the specified 500\n",
      "Created a chunk of size 973, which is longer than the specified 500\n",
      "Created a chunk of size 892, which is longer than the specified 500\n",
      "Created a chunk of size 1138, which is longer than the specified 500\n",
      "Created a chunk of size 557, which is longer than the specified 500\n",
      "Created a chunk of size 615, which is longer than the specified 500\n",
      "Created a chunk of size 909, which is longer than the specified 500\n",
      "Created a chunk of size 627, which is longer than the specified 500\n",
      "Created a chunk of size 551, which is longer than the specified 500\n",
      "Created a chunk of size 717, which is longer than the specified 500\n",
      "Created a chunk of size 944, which is longer than the specified 500\n",
      "Created a chunk of size 911, which is longer than the specified 500\n",
      "Created a chunk of size 625, which is longer than the specified 500\n",
      "Created a chunk of size 637, which is longer than the specified 500\n",
      "Created a chunk of size 754, which is longer than the specified 500\n",
      "Created a chunk of size 991, which is longer than the specified 500\n",
      "Created a chunk of size 589, which is longer than the specified 500\n",
      "Created a chunk of size 739, which is longer than the specified 500\n",
      "Created a chunk of size 541, which is longer than the specified 500\n",
      "Created a chunk of size 522, which is longer than the specified 500\n",
      "Created a chunk of size 700, which is longer than the specified 500\n",
      "Created a chunk of size 959, which is longer than the specified 500\n",
      "Created a chunk of size 851, which is longer than the specified 500\n",
      "Created a chunk of size 542, which is longer than the specified 500\n",
      "Created a chunk of size 728, which is longer than the specified 500\n",
      "Created a chunk of size 633, which is longer than the specified 500\n",
      "Created a chunk of size 758, which is longer than the specified 500\n",
      "Created a chunk of size 1040, which is longer than the specified 500\n",
      "Created a chunk of size 667, which is longer than the specified 500\n",
      "Created a chunk of size 656, which is longer than the specified 500\n",
      "Created a chunk of size 784, which is longer than the specified 500\n",
      "Created a chunk of size 896, which is longer than the specified 500\n",
      "Created a chunk of size 1479, which is longer than the specified 500\n",
      "Created a chunk of size 766, which is longer than the specified 500\n",
      "Created a chunk of size 513, which is longer than the specified 500\n",
      "Created a chunk of size 966, which is longer than the specified 500\n",
      "Created a chunk of size 645, which is longer than the specified 500\n",
      "Created a chunk of size 502, which is longer than the specified 500\n",
      "Created a chunk of size 1095, which is longer than the specified 500\n",
      "Created a chunk of size 1448, which is longer than the specified 500\n",
      "Created a chunk of size 940, which is longer than the specified 500\n",
      "Created a chunk of size 679, which is longer than the specified 500\n",
      "Created a chunk of size 552, which is longer than the specified 500\n",
      "Created a chunk of size 513, which is longer than the specified 500\n",
      "Created a chunk of size 839, which is longer than the specified 500\n",
      "Created a chunk of size 549, which is longer than the specified 500\n",
      "Created a chunk of size 553, which is longer than the specified 500\n",
      "Created a chunk of size 1140, which is longer than the specified 500\n",
      "Created a chunk of size 543, which is longer than the specified 500\n",
      "Created a chunk of size 1388, which is longer than the specified 500\n",
      "Created a chunk of size 2019, which is longer than the specified 500\n",
      "Created a chunk of size 788, which is longer than the specified 500\n",
      "Created a chunk of size 818, which is longer than the specified 500\n",
      "Created a chunk of size 1120, which is longer than the specified 500\n",
      "Created a chunk of size 757, which is longer than the specified 500\n",
      "Created a chunk of size 893, which is longer than the specified 500\n",
      "Created a chunk of size 702, which is longer than the specified 500\n",
      "Created a chunk of size 636, which is longer than the specified 500\n",
      "Created a chunk of size 851, which is longer than the specified 500\n",
      "Created a chunk of size 994, which is longer than the specified 500\n",
      "Created a chunk of size 1295, which is longer than the specified 500\n",
      "Created a chunk of size 733, which is longer than the specified 500\n",
      "Created a chunk of size 871, which is longer than the specified 500\n",
      "Created a chunk of size 593, which is longer than the specified 500\n",
      "Created a chunk of size 517, which is longer than the specified 500\n",
      "Created a chunk of size 810, which is longer than the specified 500\n",
      "Created a chunk of size 942, which is longer than the specified 500\n",
      "Created a chunk of size 807, which is longer than the specified 500\n",
      "Created a chunk of size 604, which is longer than the specified 500\n",
      "Created a chunk of size 657, which is longer than the specified 500\n",
      "Created a chunk of size 922, which is longer than the specified 500\n",
      "Created a chunk of size 1252, which is longer than the specified 500\n",
      "Created a chunk of size 676, which is longer than the specified 500\n",
      "Created a chunk of size 575, which is longer than the specified 500\n",
      "Created a chunk of size 908, which is longer than the specified 500\n",
      "Created a chunk of size 690, which is longer than the specified 500\n",
      "Created a chunk of size 1416, which is longer than the specified 500\n",
      "Created a chunk of size 614, which is longer than the specified 500\n",
      "Created a chunk of size 881, which is longer than the specified 500\n",
      "Created a chunk of size 519, which is longer than the specified 500\n",
      "Created a chunk of size 971, which is longer than the specified 500\n",
      "Created a chunk of size 1193, which is longer than the specified 500\n",
      "Created a chunk of size 527, which is longer than the specified 500\n",
      "Created a chunk of size 1028, which is longer than the specified 500\n",
      "Created a chunk of size 1201, which is longer than the specified 500\n",
      "Created a chunk of size 507, which is longer than the specified 500\n",
      "Created a chunk of size 1001, which is longer than the specified 500\n",
      "Created a chunk of size 1078, which is longer than the specified 500\n",
      "Created a chunk of size 1355, which is longer than the specified 500\n",
      "Created a chunk of size 834, which is longer than the specified 500\n",
      "Created a chunk of size 1288, which is longer than the specified 500\n",
      "Created a chunk of size 746, which is longer than the specified 500\n",
      "Created a chunk of size 656, which is longer than the specified 500\n",
      "Created a chunk of size 940, which is longer than the specified 500\n",
      "Created a chunk of size 532, which is longer than the specified 500\n",
      "Created a chunk of size 711, which is longer than the specified 500\n",
      "Created a chunk of size 892, which is longer than the specified 500\n",
      "Created a chunk of size 524, which is longer than the specified 500\n",
      "Created a chunk of size 576, which is longer than the specified 500\n",
      "Created a chunk of size 601, which is longer than the specified 500\n",
      "Created a chunk of size 656, which is longer than the specified 500\n",
      "Created a chunk of size 564, which is longer than the specified 500\n",
      "Created a chunk of size 899, which is longer than the specified 500\n",
      "Created a chunk of size 532, which is longer than the specified 500\n",
      "Created a chunk of size 590, which is longer than the specified 500\n",
      "Created a chunk of size 830, which is longer than the specified 500\n",
      "Created a chunk of size 1058, which is longer than the specified 500\n",
      "Created a chunk of size 1105, which is longer than the specified 500\n",
      "Created a chunk of size 933, which is longer than the specified 500\n",
      "Created a chunk of size 1093, which is longer than the specified 500\n",
      "Created a chunk of size 749, which is longer than the specified 500\n",
      "Created a chunk of size 641, which is longer than the specified 500\n",
      "Created a chunk of size 591, which is longer than the specified 500\n",
      "Created a chunk of size 1306, which is longer than the specified 500\n",
      "Created a chunk of size 1011, which is longer than the specified 500\n",
      "Created a chunk of size 1115, which is longer than the specified 500\n",
      "Created a chunk of size 1034, which is longer than the specified 500\n",
      "Created a chunk of size 542, which is longer than the specified 500\n",
      "Created a chunk of size 866, which is longer than the specified 500\n",
      "Created a chunk of size 868, which is longer than the specified 500\n",
      "Created a chunk of size 1153, which is longer than the specified 500\n",
      "Created a chunk of size 2108, which is longer than the specified 500\n",
      "Created a chunk of size 940, which is longer than the specified 500\n",
      "Created a chunk of size 530, which is longer than the specified 500\n",
      "Created a chunk of size 503, which is longer than the specified 500\n",
      "Created a chunk of size 520, which is longer than the specified 500\n",
      "Created a chunk of size 849, which is longer than the specified 500\n",
      "Created a chunk of size 585, which is longer than the specified 500\n",
      "Created a chunk of size 538, which is longer than the specified 500\n",
      "Created a chunk of size 938, which is longer than the specified 500\n",
      "Created a chunk of size 502, which is longer than the specified 500\n",
      "Created a chunk of size 757, which is longer than the specified 500\n",
      "Created a chunk of size 736, which is longer than the specified 500\n",
      "Created a chunk of size 821, which is longer than the specified 500\n",
      "Created a chunk of size 1478, which is longer than the specified 500\n",
      "Created a chunk of size 948, which is longer than the specified 500\n",
      "Created a chunk of size 553, which is longer than the specified 500\n",
      "Created a chunk of size 894, which is longer than the specified 500\n",
      "Created a chunk of size 597, which is longer than the specified 500\n",
      "Created a chunk of size 810, which is longer than the specified 500\n",
      "Created a chunk of size 570, which is longer than the specified 500\n",
      "Created a chunk of size 627, which is longer than the specified 500\n",
      "Created a chunk of size 540, which is longer than the specified 500\n",
      "Created a chunk of size 857, which is longer than the specified 500\n",
      "Created a chunk of size 947, which is longer than the specified 500\n",
      "Created a chunk of size 1226, which is longer than the specified 500\n",
      "Created a chunk of size 633, which is longer than the specified 500\n",
      "Created a chunk of size 569, which is longer than the specified 500\n",
      "Created a chunk of size 601, which is longer than the specified 500\n",
      "Created a chunk of size 679, which is longer than the specified 500\n",
      "Created a chunk of size 1235, which is longer than the specified 500\n",
      "Created a chunk of size 773, which is longer than the specified 500\n",
      "Created a chunk of size 734, which is longer than the specified 500\n",
      "Created a chunk of size 567, which is longer than the specified 500\n",
      "Created a chunk of size 633, which is longer than the specified 500\n",
      "Created a chunk of size 641, which is longer than the specified 500\n",
      "Created a chunk of size 505, which is longer than the specified 500\n",
      "Created a chunk of size 816, which is longer than the specified 500\n",
      "Created a chunk of size 799, which is longer than the specified 500\n",
      "Created a chunk of size 806, which is longer than the specified 500\n",
      "Created a chunk of size 516, which is longer than the specified 500\n",
      "Created a chunk of size 923, which is longer than the specified 500\n",
      "Created a chunk of size 622, which is longer than the specified 500\n",
      "Created a chunk of size 1254, which is longer than the specified 500\n",
      "Created a chunk of size 548, which is longer than the specified 500\n",
      "Created a chunk of size 658, which is longer than the specified 500\n",
      "Created a chunk of size 1363, which is longer than the specified 500\n",
      "Created a chunk of size 501, which is longer than the specified 500\n",
      "Created a chunk of size 559, which is longer than the specified 500\n",
      "Created a chunk of size 649, which is longer than the specified 500\n",
      "Created a chunk of size 588, which is longer than the specified 500\n",
      "Created a chunk of size 585, which is longer than the specified 500\n",
      "Created a chunk of size 1844, which is longer than the specified 500\n",
      "Created a chunk of size 1471, which is longer than the specified 500\n",
      "Created a chunk of size 705, which is longer than the specified 500\n",
      "Created a chunk of size 501, which is longer than the specified 500\n",
      "Created a chunk of size 624, which is longer than the specified 500\n",
      "Created a chunk of size 630, which is longer than the specified 500\n",
      "Created a chunk of size 632, which is longer than the specified 500\n",
      "Created a chunk of size 1059, which is longer than the specified 500\n",
      "Created a chunk of size 1366, which is longer than the specified 500\n",
      "Created a chunk of size 532, which is longer than the specified 500\n",
      "Created a chunk of size 687, which is longer than the specified 500\n",
      "Created a chunk of size 945, which is longer than the specified 500\n",
      "Created a chunk of size 504, which is longer than the specified 500\n",
      "Created a chunk of size 535, which is longer than the specified 500\n",
      "Created a chunk of size 976, which is longer than the specified 500\n",
      "Created a chunk of size 503, which is longer than the specified 500\n",
      "Created a chunk of size 1204, which is longer than the specified 500\n",
      "Created a chunk of size 584, which is longer than the specified 500\n",
      "Created a chunk of size 565, which is longer than the specified 500\n",
      "Created a chunk of size 507, which is longer than the specified 500\n",
      "Created a chunk of size 547, which is longer than the specified 500\n",
      "Created a chunk of size 655, which is longer than the specified 500\n",
      "Created a chunk of size 884, which is longer than the specified 500\n",
      "Created a chunk of size 859, which is longer than the specified 500\n",
      "Created a chunk of size 731, which is longer than the specified 500\n",
      "Created a chunk of size 643, which is longer than the specified 500\n",
      "Created a chunk of size 1228, which is longer than the specified 500\n",
      "Created a chunk of size 859, which is longer than the specified 500\n",
      "Created a chunk of size 944, which is longer than the specified 500\n",
      "Created a chunk of size 750, which is longer than the specified 500\n",
      "Created a chunk of size 777, which is longer than the specified 500\n",
      "Created a chunk of size 702, which is longer than the specified 500\n",
      "Created a chunk of size 1769, which is longer than the specified 500\n",
      "Created a chunk of size 734, which is longer than the specified 500\n",
      "Created a chunk of size 1257, which is longer than the specified 500\n",
      "Created a chunk of size 602, which is longer than the specified 500\n",
      "Created a chunk of size 1767, which is longer than the specified 500\n",
      "Created a chunk of size 2463, which is longer than the specified 500\n",
      "Created a chunk of size 774, which is longer than the specified 500\n",
      "Created a chunk of size 551, which is longer than the specified 500\n",
      "Created a chunk of size 731, which is longer than the specified 500\n",
      "Created a chunk of size 686, which is longer than the specified 500\n",
      "Created a chunk of size 1142, which is longer than the specified 500\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1497"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_text_splitters import SpacyTextSplitter\n",
    "\n",
    "text_splitter = SpacyTextSplitter(chunk_size=500)\n",
    "\n",
    "sentence_chunks = text_splitter.split_text(text)\n",
    "len(sentence_chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "id": "33e53132-aa2b-414c-b1ea-b813876cabae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------\n",
      "First Chunk:\n",
      "Uber \n",
      "2021 \n",
      "Annual ReportUber’s Mission\n",
      "We reimagine the way the world moves for the better\n",
      "We are Uber.\n",
      "\n",
      "The go-getters.\n",
      "\n",
      "The kind of people who are relentless about our mission to help people go anywhere and get anything  and earn their way . \n",
      "\n",
      "\n",
      "Movement is what we  power .\n",
      "\n",
      "It’s our lifeblood.\n",
      "\n",
      "It runs through our veins.\n",
      "\n",
      "It’s what gets us out of bed each morning.\n",
      "\n",
      "It pushes us to constantly reimagine \n",
      "how we can move better.\n",
      "\n",
      "For you.\n",
      "\n",
      "For all the places you want to go.\n",
      "--------------\n",
      "Second Chunk:\n",
      "It’s our lifeblood.\n",
      "\n",
      "It runs through our veins.\n",
      "\n",
      "It’s what gets us out of bed each morning.\n",
      "\n",
      "It pushes us to constantly reimagine \n",
      "how we can move better.\n",
      "\n",
      "For you.\n",
      "\n",
      "For all the places you want to go.\n",
      "\n",
      "For all the things you want to get.\n",
      "\n",
      "For all the ways you want to earn.\n",
      "\n",
      "Across the entire \n",
      "world.\n",
      "\n",
      "In real time .\n",
      "\n",
      "At the incredible speed of now.\n",
      "\n",
      "\n",
      "Our Values\n",
      "Our values reflect who we are and where we’re going.\n",
      "--------------\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n--------------\")\n",
    "print(\"First Chunk:\\n\"+sentence_chunks[0]+ \"\\n--------------\")\n",
    "print(\"Second Chunk:\\n\"+sentence_chunks[1]+ \"\\n--------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "id": "c3281104-8190-4fef-b1f8-28ac00909753",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_text_splitters import NLTKTextSplitter\n",
    "\n",
    "text_splitter = NLTKTextSplitter(chunk_size=500)\n",
    "nltk_chunks = text_splitter.split_text(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "id": "e1240b57-ce3a-4920-b2db-6a52008541a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------\n",
      "First Chunk:\n",
      "Uber \n",
      "2021 \n",
      "Annual ReportUber’s Mission\n",
      "We reimagine the way the world moves for the better\n",
      "We are Uber.\n",
      "\n",
      "The go-getters.\n",
      "\n",
      "The kind of people who are relentless about our mission to help people go anywhere and get anything  and earn their way . \n",
      "\n",
      "\n",
      "Movement is what we  power .\n",
      "\n",
      "It’s our lifeblood.\n",
      "\n",
      "It runs through our veins.\n",
      "\n",
      "It’s what gets us out of bed each morning.\n",
      "\n",
      "It pushes us to constantly reimagine \n",
      "how we can move better.\n",
      "\n",
      "For you.\n",
      "\n",
      "For all the places you want to go.\n",
      "--------------\n",
      "Second Chunk:\n",
      "It’s our lifeblood.\n",
      "\n",
      "It runs through our veins.\n",
      "\n",
      "It’s what gets us out of bed each morning.\n",
      "\n",
      "It pushes us to constantly reimagine \n",
      "how we can move better.\n",
      "\n",
      "For you.\n",
      "\n",
      "For all the places you want to go.\n",
      "\n",
      "For all the things you want to get.\n",
      "\n",
      "For all the ways you want to earn.\n",
      "\n",
      "Across the entire \n",
      "world.\n",
      "\n",
      "In real time .\n",
      "\n",
      "At the incredible speed of now.\n",
      "\n",
      "\n",
      "Our Values\n",
      "Our values reflect who we are and where we’re going.\n",
      "--------------\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n--------------\")\n",
    "print(\"First Chunk:\\n\"+sentence_chunks[0]+ \"\\n--------------\")\n",
    "print(\"Second Chunk:\\n\"+sentence_chunks[1]+ \"\\n--------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "id": "c3a173c5-d033-4e6f-99bb-57830df6f51f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------\n",
      "First Chunk:\n",
      "Hi this is Jim  \n",
      "Hi this is Joe\n",
      "--------------\n",
      "Second Chunk:\n",
      "Hi this is Lance\n",
      "--------------\n"
     ]
    }
   ],
   "source": [
    "from langchain_text_splitters import MarkdownHeaderTextSplitter\n",
    "markdown_document = \"# Foo\\n\\n    ## Bar\\n\\nHi this is Jim\\n\\nHi this is Joe\\n\\n ### Boo \\n\\n Hi this is Lance \\n\\n ## Baz\\n\\n Hi this is Molly\"\n",
    "\n",
    "headers_to_split_on = [\n",
    "   (\"#\", \"Header 1\"),\n",
    "   (\"##\", \"Header 2\"),\n",
    "   (\"###\", \"Header 3\"),\n",
    "]\n",
    "\n",
    "markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)\n",
    "md_header_splits = markdown_splitter.split_text(markdown_document)\n",
    "print(\"\\n--------------\")\n",
    "print(\"First Chunk:\\n\"+md_header_splits[0].page_content+ \"\\n--------------\")\n",
    "print(\"Second Chunk:\\n\"+md_header_splits[1].page_content+ \"\\n--------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba2fb80d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -qU langchain_experimental"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "id": "d94ee38c-4af8-4046-a508-b2a9a4962d32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------\n",
      "First Chunk:\n",
      "Uber \n",
      "2021 \n",
      "Annual ReportUber’s Mission\n",
      "We reimagine the way the world moves for the better\n",
      "We are Uber. The go-getters.\n",
      "--------------\n",
      "Second Chunk:\n",
      "The kind of people who are relentless about our mission to help people go anywhere and get anything  and earn their way . Movement is what we  power .\n",
      "--------------\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from langchain_experimental.text_splitter import SemanticChunker\n",
    "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
    "\n",
    "text_splitter = SemanticChunker(HuggingFaceEmbeddings())\n",
    "\n",
    "semantic_chunks = text_splitter.split_text(text)\n",
    "print(\"\\n--------------\")\n",
    "print(\"First Chunk:\\n\"+semantic_chunks[0]+ \"\\n--------------\")\n",
    "print(\"Second Chunk:\\n\"+semantic_chunks[1]+ \"\\n--------------\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbdcb0ed-9a2f-4b21-9120-8fea6d25cc9a",
   "metadata": {},
   "source": [
    "# Advanced RAG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "009a6d76-c904-4739-a895-7f74e2210357",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_uri= \"s3://ashwin-partner-bucket/fashion_dataset.jsonl\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c1f8bd3-b89a-4915-a8d1-041b8aa89509",
   "metadata": {},
   "source": [
    "# Sample Dataset with OpenAIEmbedding already included in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "201ea68a-f3fb-40bd-91ea-59e0be88edde",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ageGroup</th>\n",
       "      <th>link</th>\n",
       "      <th>brandName</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>gender</th>\n",
       "      <th>subCategory</th>\n",
       "      <th>masterCategory</th>\n",
       "      <th>season</th>\n",
       "      <th>articleType</th>\n",
       "      <th>baseColour</th>\n",
       "      <th>id</th>\n",
       "      <th>openAIVec</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adults-Women</td>\n",
       "      <td>http://assets.myntassets.com/v1/images/style/p...</td>\n",
       "      <td>Inc 5</td>\n",
       "      <td>1390.0</td>\n",
       "      <td>Inc. 5 Women Casual White Flats</td>\n",
       "      <td>Women</td>\n",
       "      <td>Shoes</td>\n",
       "      <td>Footwear</td>\n",
       "      <td>Winter</td>\n",
       "      <td>Heels</td>\n",
       "      <td>White</td>\n",
       "      <td>22275</td>\n",
       "      <td>[-0.016259265699646003, -0.016057870922684, -0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Adults-Women</td>\n",
       "      <td>http://assets.myntassets.com/v1/images/style/p...</td>\n",
       "      <td>French Connection</td>\n",
       "      <td>3999.0</td>\n",
       "      <td>French Connection Women Black Sling Bag</td>\n",
       "      <td>Women</td>\n",
       "      <td>Bags</td>\n",
       "      <td>Accessories</td>\n",
       "      <td>Summer</td>\n",
       "      <td>Handbags</td>\n",
       "      <td>Black</td>\n",
       "      <td>42874</td>\n",
       "      <td>[-0.022201561751436002, 0.006381784873631001, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kids-Girls</td>\n",
       "      <td>http://assets.myntassets.com/v1/images/style/p...</td>\n",
       "      <td>Q&amp;Q</td>\n",
       "      <td>625.0</td>\n",
       "      <td>Q&amp;Q Kids Girls White Dial Analog Watch</td>\n",
       "      <td>Girls</td>\n",
       "      <td>Watches</td>\n",
       "      <td>Accessories</td>\n",
       "      <td>Winter</td>\n",
       "      <td>Watches</td>\n",
       "      <td>Pink</td>\n",
       "      <td>49888</td>\n",
       "      <td>[-0.001154974972115, 0.010144626266777, -7.218...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ageGroup                                               link  \\\n",
       "0  Adults-Women  http://assets.myntassets.com/v1/images/style/p...   \n",
       "1  Adults-Women  http://assets.myntassets.com/v1/images/style/p...   \n",
       "2    Kids-Girls  http://assets.myntassets.com/v1/images/style/p...   \n",
       "\n",
       "           brandName   price                                    title gender  \\\n",
       "0              Inc 5  1390.0          Inc. 5 Women Casual White Flats  Women   \n",
       "1  French Connection  3999.0  French Connection Women Black Sling Bag  Women   \n",
       "2                Q&Q   625.0   Q&Q Kids Girls White Dial Analog Watch  Girls   \n",
       "\n",
       "  subCategory masterCategory  season articleType baseColour     id  \\\n",
       "0       Shoes       Footwear  Winter       Heels      White  22275   \n",
       "1        Bags    Accessories  Summer    Handbags      Black  42874   \n",
       "2     Watches    Accessories  Winter     Watches       Pink  49888   \n",
       "\n",
       "                                           openAIVec  \n",
       "0  [-0.016259265699646003, -0.016057870922684, -0...  \n",
       "1  [-0.022201561751436002, 0.006381784873631001, ...  \n",
       "2  [-0.001154974972115, 0.010144626266777, -7.218...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import s3fs\n",
    "import boto3\n",
    "df = pd.read_json(s3_uri, orient=\"records\", lines=True)\n",
    "df[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c59cdbf-fa3c-4d66-a611-d9291a519c87",
   "metadata": {},
   "source": [
    "# Insert Data to MongoDB Atlas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f0cf8bac-cd34-4cd8-a907-88f2c7c98494",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "import certifi\n",
    "mongo_client = MongoClient(os.environ[\"MONGODB_CONNECTION_STR\"], tlsCAFile=certifi.where())\n",
    "# Upload documents along with vector embeddings to MongoDB Atlas Collection\n",
    "col = mongo_client[\"search\"][\"catalog_final_myn\"]\n",
    "col.insert_many(df.to_dict(orient=\"records\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "543bc1e6-cd12-419a-9e40-557432ad97cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from langchain_core.output_parsers import JsonOutputParser # type: ignore\n",
    "from langchain_core.prompts import PromptTemplate # type: ignore\n",
    "from langchain_core.pydantic_v1 import BaseModel, Field # type: ignore\n",
    "from langchain_openai import ChatOpenAI # type: ignore\n",
    "\n",
    "from langchain_openai.embeddings import OpenAIEmbeddings # type: ignore\n",
    "from langchain_mongodb.vectorstores import MongoDBAtlasVectorSearch # type: ignore\n",
    "\n",
    "\n",
    "from pymongo import MongoClient # type: ignore\n",
    "from typing import List\n",
    "from itertools import chain\n",
    "import certifi # type: ignore\n",
    "import os\n",
    "from dotenv import load_dotenv # type: ignore\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "from functools import lru_cache\n",
    "\n",
    "@lru_cache\n",
    "def get_openai_emb_transformers():\n",
    "    \"\"\"\n",
    "    Returns an instance of OpenAIEmbeddings for OpenAI transformer models.\n",
    "    \n",
    "    This function creates and returns an instance of the OpenAIEmbeddings class,\n",
    "    which provides access to OpenAI transformer models for natural language processing.\n",
    "    The instance is cached using the lru_cache decorator for efficient reuse.\n",
    "    \n",
    "    Returns:\n",
    "        embeddings (OpenAIEmbeddings): An instance of the OpenAIEmbeddings class.\n",
    "    \"\"\"\n",
    "    embeddings = OpenAIEmbeddings()\n",
    "    return embeddings\n",
    "\n",
    "@lru_cache\n",
    "def get_vector_store():\n",
    "    \"\"\"\n",
    "    Retrieves the vector store for MongoDB Atlas.\n",
    "\n",
    "    Returns:\n",
    "        MongoDBAtlasVectorSearch: The vector store object.\n",
    "    \"\"\"\n",
    "    vs = MongoDBAtlasVectorSearch(collection=col, embedding=get_openai_emb_transformers(), index_name=\"vector_index_openAi_cosine\", embedding_key=\"openAIVec\", text_key=\"title\")\n",
    "    return vs\n",
    "\n",
    "@lru_cache(10)\n",
    "def get_conversation_chain_conv():\n",
    "    \"\"\"\n",
    "    Retrieves a conversation chain model for chat conversations.\n",
    "\n",
    "    Returns:\n",
    "        ChatOpenAI: The conversation chain model for chat conversations.\n",
    "    \"\"\"\n",
    "    llm = ChatOpenAI(model=\"gpt-3.5-turbo\", temperature=0.2, max_tokens=2048)\n",
    "    # chain = ConversationChain(llm=llm, memory=ConversationBufferWindowMemory(k=5))\n",
    "    return llm\n",
    "\n",
    "\n",
    "# Define your desired data structure.\n",
    "class ProductRecoStatus(BaseModel):\n",
    "    \"\"\"\n",
    "    Represents the status of product recommendations.\n",
    "\n",
    "    Attributes:\n",
    "        relevancy_status (bool): Product recommendation status conditioned on the context of the input query.\n",
    "                                 True if the query is related to purchasing fashion clothing and/or accessories.\n",
    "                                 False otherwise.\n",
    "        recommendations (List[str]): List of recommended product titles based on the input query context and\n",
    "                                     if the relevancy_status is True.\n",
    "    \"\"\"\n",
    "    relevancy_status: bool = Field(description=\"Product recommendation status is conditioned on the fact if the context of input query is to purchase a fashion clothing and or fashion accessories.\")\n",
    "    recommendations: List[str] = Field(description=\"list of recommended product titles based on the input query context and if recommendation_status is true.\")\n",
    "\n",
    "\n",
    "class Product(BaseModel):\n",
    "    \"\"\"\n",
    "    Represents a product.\n",
    "\n",
    "    Attributes:\n",
    "        title (str): Title of the product.\n",
    "        baseColour (List[str]): List of base colours of the product.\n",
    "        gender (List[str]): List of genders the product is targeted for.\n",
    "        articleType (str): Type of the article.\n",
    "        mfg_brand_name (str): Manufacturer or brand name of the product.\n",
    "    \"\"\"\n",
    "    title: str = Field(description=\"Title of the product.\")\n",
    "    baseColour: List[str] = Field(description=\"List of base colours of the product.\")\n",
    "    gender: List[str] = Field(description=\"List of genders the product is targeted for.\")\n",
    "    articleType: str = Field(description=\"Type of the article.\")\n",
    "    mfg_brand_name: str = Field(description=\"Manufacturer or brand name of the product.\")\n",
    "\n",
    "\n",
    "class Recommendations(BaseModel):\n",
    "    \"\"\"\n",
    "    Represents a set of recommendations for products and a message to the user.\n",
    "\n",
    "    Attributes:\n",
    "        products (List[Product]): List of recommended products.\n",
    "        message (str): Message to the user and context of the chat history summary.\n",
    "    \"\"\"\n",
    "    products: List[Product] = Field(description=\"List of recommended products.\")\n",
    "    message: str = Field(description=\"Message to the user and context of the chat history summary.\")\n",
    "\n",
    "\n",
    "reco_status_parser = JsonOutputParser(pydantic_object=ProductRecoStatus)\n",
    "\n",
    "reco_status_prompt = PromptTemplate(\n",
    "    template=\"You are AI assistant tasked at identifying if there is a product purchase intent in the query and providing suitable fashion recommendations.\\n{format_instructions}\\n{query}\\n\\\n",
    "        #Chat History Summary: {chat_history}\\n\\nBased on the context of the query, please provide the relevancy status and list of recommended products.\",\n",
    "    input_variables=[\"query\", \"chat_history\"],\n",
    "    partial_variables={\"format_instructions\": reco_status_parser.get_format_instructions()},\n",
    ")\n",
    "\n",
    "reco_parser = JsonOutputParser(pydantic_object=Recommendations)\n",
    "reco_prompt = PromptTemplate(\n",
    "    input_variables=[\"question\", \"recommendations\", \"chat_history\"],\n",
    "    partial_variables={\"format_instructions\": reco_parser.get_format_instructions()},\n",
    "    template=\"\\n User query:{question} \\n Chat Summary: {chat_history} \\n Rank and suggest me suitable products for creating grouped product recommendations given all product recommendations below feature atleast one product for each articleType \\n {recommendations} \\n show output in {format_instructions} for top 10 products\"\n",
    ")\n",
    "\n",
    "\n",
    "def get_product_reco_status(query: str, chat_history: List[str] = []):\n",
    "    \"\"\"\n",
    "    Retrieves the recommendation status for a product based on the given query and chat history.\n",
    "\n",
    "    Args:\n",
    "        query (str): The query to be used for retrieving the recommendation status.\n",
    "        chat_history (List[str]): The chat history containing previous conversations.\n",
    "\n",
    "    Returns:\n",
    "        The response containing the recommendation status.\n",
    "    \"\"\"\n",
    "    llm = get_conversation_chain_conv()\n",
    "    chain = reco_status_prompt | llm | reco_status_parser\n",
    "    resp = chain.invoke({\"query\": query, \"chat_history\": chat_history})\n",
    "    return resp\n",
    "\n",
    "def get_sorted_results(product_recommendations):\n",
    "    all_titles = [rec['title'] for rec in product_recommendations['products']]\n",
    "    results = list(col.find({\"title\": {\"$in\":all_titles}}, {\"_id\": 0 , \"id\":1, \"title\": 1, \"price\": 1, \"baseColour\": 1, \"articleType\": 1, \"gender\": 1, \"link\" : 1, \"mfg_brand_name\": 1}))\n",
    "    sorted_results = []\n",
    "    for title in all_titles:\n",
    "        for result in results:\n",
    "            if result['title'] == title:\n",
    "                sorted_results.append(result)\n",
    "                break\n",
    "    return sorted_results\n",
    "\n",
    "def get_product_recommendations(query: str, reco_queries: List[str], chat_history: List[str]=[]):\n",
    "    \"\"\"\n",
    "    Retrieves product recommendations based on the given query and chat history.\n",
    "\n",
    "    Args:\n",
    "        query (str): The query string for the recommendation.\n",
    "        chat_history (List[str]): The list of previous chat messages.\n",
    "        filter_query (dict): The filter query to apply during the recommendation retrieval.\n",
    "        reco_queries (List[str]): The list of recommendation queries.\n",
    "\n",
    "    Returns:\n",
    "        dict: The response containing the recommendations.\n",
    "\n",
    "    \"\"\"\n",
    "    vectorstore = get_vector_store()\n",
    "    retr = vectorstore.as_retriever(search_kwargs={\"k\": 10})\n",
    "    all_recommendations = list(chain(*retr.batch(reco_queries)))\n",
    "    llm = get_conversation_chain_conv()\n",
    "    llm_chain = reco_prompt | llm | reco_parser\n",
    "    resp = llm_chain.invoke({\"question\": query, \"chat_history\": chat_history, \"recommendations\": [v.page_content for v in all_recommendations]})\n",
    "    resp = get_sorted_results(resp)\n",
    "    return resp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "fcdb6f96-0a2c-463c-beaf-8427116c9648",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'relevancy_status': True, 'recommendations': ['Floral Wrap Dress', 'Off-Shoulder Maxi Dress', 'Lace Fit and Flare Dress', 'Ruffled Hem Shift Dress', 'Denim Shirt Dress']}\n",
      "[{'link': 'http://assets.myntassets.com/v1/images/style/properties/ebb8a69f6e56cf47f9fefd3ac23cfe03_images.jpg', 'price': 690.0, 'title': 'Femella Women Floral Red Dress', 'gender': 'Women', 'mfg_brand_name': 'Femella', 'articleType': 'Dresses', 'baseColour': 'Red', 'id': '39217'}, {'link': 'http://assets.myntassets.com/v1/images/style/properties/730a44ed829f9310beb97f583a960262_images.jpg', 'price': 3800.0, 'title': 'Forever New Women Floral Purple Dress', 'gender': 'Women', 'mfg_brand_name': 'Forever New', 'articleType': 'Dresses', 'baseColour': 'Purple', 'id': '8484'}, {'link': 'http://assets.myntassets.com/v1/images/style/properties/4e056a5231b2f0f427e2b9f2d45a2662_images.jpg', 'price': 1849.0, 'title': 'Mineral Women Floral Orange Dress', 'gender': 'Women', 'mfg_brand_name': 'Mineral', 'articleType': 'Dresses', 'baseColour': 'Orange', 'id': '37912'}, {'link': 'http://assets.myntassets.com/v1/images/style/properties/498885fabd8f2adb98e3563814a7be15_images.jpg', 'price': 1999.0, 'title': 'Arrow Woman Multi Coloured Floral Dress', 'gender': 'Women', 'mfg_brand_name': 'Arrow Woman', 'articleType': 'Dresses', 'baseColour': 'Multi', 'id': '23050'}, {'link': 'http://assets.myntassets.com/v1/images/style/properties/Tonga-White-Floral-Design-Dress_c8060cca2e01d587c96badd11844ecdd_images.jpg', 'price': 2190.0, 'title': 'Tonga White Floral Design Dress', 'gender': 'Women', 'mfg_brand_name': 'Tonga', 'articleType': 'Dresses', 'baseColour': 'White', 'id': '58499'}, {'link': 'http://assets.myntassets.com/v1/images/style/properties/a4be477154c1abe180dd8875e082ad6d_images.jpg', 'price': 599.0, 'title': 'Doodle Kids Girls Navy Blue Floral Print Dress', 'gender': 'Girls', 'mfg_brand_name': 'Doodle', 'articleType': 'Dresses', 'baseColour': 'Navy Blue', 'id': '23621'}, {'link': 'http://assets.myntassets.com/v1/images/style/properties/33abe02e37b16010b068ebc37de2b42a_images.jpg', 'price': 4000.0, 'title': 'Forever New Women Blossom Silk Cream Dress', 'gender': 'Women', 'mfg_brand_name': 'Forever New', 'articleType': 'Dresses', 'baseColour': 'Cream', 'id': '8479'}, {'link': 'http://assets.myntassets.com/v1/images/style/properties/555c6808b415c53b60a6c24cbc7b4274_images.jpg', 'price': 4200.0, 'title': 'Forever New Women GeoPrint Multi Coloured Dress', 'gender': 'Women', 'mfg_brand_name': 'Forever New', 'articleType': 'Dresses', 'baseColour': 'Multi', 'id': '8500'}, {'link': 'http://assets.myntassets.com/v1/images/style/properties/Femella-Women-Multi-Coloured-Dress_f866e35c53c70998b24e0315dfad8051_images.jpg', 'price': 890.0, 'title': 'Femella Women Multi Coloured Dress', 'gender': 'Women', 'mfg_brand_name': 'Femella', 'articleType': 'Dresses', 'baseColour': 'Multi', 'id': '59896'}, {'link': 'http://assets.myntassets.com/v1/images/style/properties/c1283d633e7469b32f8f16066fb511de_images.jpg', 'price': 769.0, 'title': 'Span Women Floral Printed Green Kurta', 'gender': 'Women', 'mfg_brand_name': 'Span', 'articleType': 'Kurtas', 'baseColour': 'Green', 'id': '40608'}]\n",
      "1. A flowy floral maxi dress\n",
      "2. A cute off-the-shoulder sundress\n",
      "3. A simple wrap dress in a solid color\n",
      "4. A striped shirt dress with a belt\n",
      "5. A denim overall dress\n",
      "6. A ruffled mini dress\n",
      "7. A boho-inspired peasant dress\n",
      "8. A lace-trimmed slip dress\n",
      "9. A polka dot fit and flare dress\n",
      "10. A casual jumpsuit with a cinched waist.\n"
     ]
    }
   ],
   "source": [
    "query = \"Can you suggest me some Casual dresses for date occassion with my boy friend\"\n",
    "status = get_product_reco_status(query)\n",
    "print(status)\n",
    "print(get_product_recommendations(query, reco_queries=status[\"recommendations\"], chat_history=[]))\n",
    "print(get_conversation_chain_conv().invoke(query).content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2753240a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'relevancy_status': False, 'recommendations': []}\n",
      "There are many great options for a date with your boyfriend, depending on your interests and preferences. Some ideas could include:\n",
      "\n",
      "1. A romantic dinner at a nice restaurant\n",
      "2. A picnic in the park or at the beach\n",
      "3. A movie night at home with homemade popcorn and snacks\n",
      "4. A hike or nature walk followed by a picnic\n",
      "5. A visit to a local museum or art gallery\n",
      "6. A cooking class or wine tasting experience\n",
      "7. A day trip to a nearby city or town to explore and try new restaurants\n",
      "8. A couples massage or spa day\n",
      "9. A fun activity like mini golf, bowling, or go-kart racing\n",
      "10. A concert or live music event\n",
      "\n",
      "Ultimately, the best date idea is one that you both will enjoy and that allows you to spend quality time together. Consider your boyfriend's interests and preferences when planning the date to ensure it is a memorable and enjoyable experience for both of you.\n"
     ]
    }
   ],
   "source": [
    "query = \"Where should I take my boy friend for date\" \n",
    "status = get_product_reco_status(query) \n",
    "print(status) \n",
    "print(get_conversation_chain_conv().invoke(query).content) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0711d39-cea1-466e-b3ad-a50cbcc45945",
   "metadata": {},
   "source": [
    "There are many great options for a date with your boyfriend, depending on your interests and preferences. Some ideas could include: \n",
    " \n",
    "1. A romantic dinner at a nice restaurant \n",
    "2. A picnic in the park or at the beach \n",
    "3. A movie night at home or at the cinema \n",
    "4. A hike or nature walk \n",
    "5. A visit to a museum or art gallery \n",
    "6. A cooking class or wine tasting \n",
    "7. A concert or live music event \n",
    "8. A day trip to a nearby city or town \n",
    "9. A couples massage or spa day \n",
    "10. A fun activity like mini golf, bowling, or go-kart racing \n",
    " \n",
    "Ultimately, the best date idea is one that you both will enjoy and that allows you to spend quality time together. Consider what you both like to do and choose an activity that will create lasting memories. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a4c5a02-1544-406b-b8fd-12fee90b07a7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
